{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import log_loss\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "train = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.1提取标签y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "train = train.drop(['interest_level'], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.2确定subsample和colsample_bytree参数范围"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.7, 0.8, 0.9], 'subsample': [0.7, 0.8, 0.9, 1.0]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample = [0.7, 0.8, 0.9, 1.0]# 第一次0.3-0.8\n",
    "colsample_bytree = [i/10.0 for i in range(7,10)]#第一次0.6-0.9\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=4, min_child_weight=0.5, missing=None, n_estimators=332,\n",
       "       n_jobs=1, nthread=None, objective='muti:softprob', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=True,\n",
       "       subsample=0.8),\n",
       "       fit_params={}, iid=True, n_jobs=1,\n",
       "       param_grid={'subsample': [0.7, 0.8, 0.9, 1.0], 'colsample_bytree': [0.7, 0.8, 0.9]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "         learning_rate = 0.1,\n",
    "         n_estimators = 332,#第一次226\n",
    "         max_depth=4,#第一次5\n",
    "         min_child_weight=0.5,#第一次1\n",
    "         gamma=0,\n",
    "         subsample=0.8,\n",
    "         colsample_bytree=0.8,\n",
    "         colsample_bylevel=0.7,\n",
    "         objective='muti:softprob',\n",
    "         seed=3)\n",
    "gridsearch3_1 = GridSearchCV(xgb3_1, param_grid= param_test3_1, scoring='neg_log_loss')\n",
    "gridsearch3_1.fit(X_train, y_train)\n",
    "#print('The Best score is %f and using %s'%(gridsearch3_1.best_score_, gridsearch3_1.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Best score is -0.588128 and using {'colsample_bytree': 0.7, 'subsample': 0.8}\n"
     ]
    }
   ],
   "source": [
    "print('The Best score is %f and using %s'%(gridsearch3_1.best_score_, gridsearch3_1.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\envs\\py3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[mean: -0.58903, std: 0.00256, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       " mean: -0.58813, std: 0.00261, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       " mean: -0.58892, std: 0.00253, params: {'colsample_bytree': 0.7, 'subsample': 0.9},\n",
       " mean: -0.59068, std: 0.00297, params: {'colsample_bytree': 0.7, 'subsample': 1.0},\n",
       " mean: -0.58946, std: 0.00220, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       " mean: -0.58841, std: 0.00311, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       " mean: -0.58932, std: 0.00269, params: {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       " mean: -0.59065, std: 0.00300, params: {'colsample_bytree': 0.8, 'subsample': 1.0},\n",
       " mean: -0.58841, std: 0.00230, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       " mean: -0.58821, std: 0.00264, params: {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       " mean: -0.58887, std: 0.00275, params: {'colsample_bytree': 0.9, 'subsample': 0.9},\n",
       " mean: -0.59027, std: 0.00306, params: {'colsample_bytree': 0.9, 'subsample': 1.0}]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gridsearch3_1.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 193.96076075,  191.61695981,  188.58678659,  182.0947485 ,\n",
       "         219.81290587,  217.8417933 ,  214.01224081,  205.33607793,\n",
       "         245.38836869,  242.78021955,  237.44224763,  224.33449793]),\n",
       " 'mean_score_time': array([ 1.04339298,  1.09139585,  1.06439416,  1.01105785,  1.08506211,\n",
       "         1.05872711,  1.03305912,  1.05072673,  1.06972798,  1.01939162,\n",
       "         1.05772718,  1.02439197]),\n",
       " 'mean_test_score': array([-0.58902798, -0.58812779, -0.58892222, -0.59067831, -0.58945924,\n",
       "        -0.58840604, -0.58932054, -0.59065091, -0.58841206, -0.58820947,\n",
       "        -0.58887134, -0.59026588]),\n",
       " 'mean_train_score': array([-0.5166508 , -0.51736585, -0.51983328, -0.52552992, -0.51479342,\n",
       "        -0.51618505, -0.51838146, -0.52531485, -0.51365664, -0.51461598,\n",
       "        -0.51725538, -0.52366913]),\n",
       " 'param_colsample_bytree': masked_array(data = [0.7 0.7 0.7 0.7 0.8 0.8 0.8 0.8 0.9 0.9 0.9 0.9],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_subsample': masked_array(data = [0.7 0.8 0.9 1.0 0.7 0.8 0.9 1.0 0.7 0.8 0.9 1.0],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': ({'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 1.0},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 1.0},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 1.0}),\n",
       " 'rank_test_score': array([ 7,  1,  6, 12,  9,  3,  8, 11,  4,  2,  5, 10]),\n",
       " 'split0_test_score': array([-0.59049966, -0.5901382 , -0.59044595, -0.59257822, -0.59040611,\n",
       "        -0.59031123, -0.59106887, -0.59241326, -0.58996113, -0.59043236,\n",
       "        -0.59091116, -0.5916162 ]),\n",
       " 'split0_train_score': array([-0.51660153, -0.51645728, -0.5196023 , -0.5249284 , -0.51455247,\n",
       "        -0.5158318 , -0.51841015, -0.52483005, -0.51319707, -0.51443489,\n",
       "        -0.51706848, -0.52304149]),\n",
       " 'split1_test_score': array([-0.58542678, -0.5844487 , -0.58536164, -0.58648693, -0.58641472,\n",
       "        -0.58401471, -0.58552175, -0.58643248, -0.58515607, -0.5844955 ,\n",
       "        -0.58498494, -0.58603266]),\n",
       " 'split1_train_score': array([-0.51706016, -0.51903641, -0.52181571, -0.52710756, -0.51606926,\n",
       "        -0.51730106, -0.5195914 , -0.52697603, -0.51547718, -0.51671342,\n",
       "        -0.51952208, -0.52503632]),\n",
       " 'split2_test_score': array([-0.59115764, -0.58979657, -0.59095921, -0.59296992, -0.59155702,\n",
       "        -0.59089232, -0.59137112, -0.59310712, -0.59011909, -0.58970064,\n",
       "        -0.59071803, -0.59314896]),\n",
       " 'split2_train_score': array([-0.5162907 , -0.51660386, -0.51808182, -0.52455379, -0.51375854,\n",
       "        -0.51542229, -0.51714284, -0.52413846, -0.51229566, -0.51269963,\n",
       "        -0.51517559, -0.52292959]),\n",
       " 'std_fit_time': array([ 0.2082047 ,  0.50067687,  1.5887535 ,  1.22059336,  0.42598389,\n",
       "         0.81671524,  0.69340867,  0.48365582,  1.11568993,  0.29536562,\n",
       "         0.57195125,  0.71816093]),\n",
       " 'std_score_time': array([ 0.01869759,  0.04747434,  0.01249972,  0.02535244,  0.01358988,\n",
       "         0.03600521,  0.01779621,  0.04620031,  0.04429206,  0.02451447,\n",
       "         0.02519835,  0.01400886]),\n",
       " 'std_test_score': array([ 0.0025606 ,  0.00260529,  0.00252646,  0.00296811,  0.00220351,\n",
       "         0.00311423,  0.00268902,  0.00299634,  0.00230327,  0.00264315,\n",
       "         0.00274927,  0.00305809]),\n",
       " 'std_train_score': array([ 0.00031606,  0.00118278,  0.00153308,  0.001126  ,  0.00095861,\n",
       "         0.00080665,  0.00099982,  0.00120809,  0.00133889,  0.00164362,\n",
       "         0.00177936,  0.00096783])}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gridsearch3_1.cv_results_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "first_time:colsample_bytree=0.8,subsample=0.8,得分0.58756"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "second_time:colsample_bytree=0.7,subsample=0.8,此时得分0.588128， 看了下发现两组参数0.8的时候得分确实比上一次训练低了点：0.58841，\n",
    "所以还是选择colsample_bytree=0.8,subsample=0.8."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
